{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对活动数据（events.csv）进行处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入数据包\n",
    "\n",
    "import numpy as np\n",
    "import scipy.sparse as ss\n",
    "import scipy.io as sio\n",
    "\n",
    "#保存数据\n",
    "import pickle\n",
    "\n",
    "#event的特征需要编码\n",
    "from utils import FeatureEng\n",
    "from sklearn.preprocessing import normalize\n",
    "#相似度/距离\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of records :3137972\n"
     ]
    }
   ],
   "source": [
    "#读取数据，并统计有多少不同的events\n",
    "\n",
    "lines = 0\n",
    "fin = open('/Users/cuiyue/Desktop/AI/第四周/作业/要求/71-57-1-1519640547/events.csv', 'rb')\n",
    "fin.readline().decode('utf-8').strip().split(',')\n",
    "# 列字段：['event_id','user_id','start_time','city','state','zip','country','lat','lng','c_1', ..., 'c_100', 'c_other']\n",
    "for line in fin:\n",
    "    cols = line.decode('utf-8').strip().split(',')\n",
    "    lines += 1\n",
    "fin.close()\n",
    "\n",
    "print('Number of records :%d' % lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "# 读取之前1.Users_Events中算好的测试集和训练集中出现过的活动\n",
    "\n",
    "eventIndex = pickle.load(open('PE_eventIndex.pkl', 'rb'))\n",
    "n_events = len(eventIndex)\n",
    "print(\"Number of events in train & test :%d\" % n_events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理events.csv --> 特征编码\n",
    "\n",
    "FE = FeatureEng()\n",
    "fin = open('/Users/cuiyue/Desktop/AI/第四周/作业/要求/71-57-1-1519640547/events.csv', 'rb')\n",
    "fin.readline().decode('utf-8').strip().split(',')\n",
    "# 列字段：['event_id','user_id','start_time','city','state','zip','country','lat','lng','c_1', ..., 'c_100', 'c_other']\n",
    "# 构建活动基本信息矩阵，包括字段：start_time, city, state, zip, country, lat, and lng   \n",
    "eventPropMatrix = ss.dok_matrix((n_events, 7))\n",
    "# 构建词频特征矩阵\n",
    "eventContMatrix = ss.dok_matrix((n_events, 101))\n",
    "\n",
    "for line in fin.readlines():\n",
    "    cols = line.decode('utf-8').strip().split(',')\n",
    "   # cols = cols.encode('utf-8')\n",
    "    eventId = str(cols[0])\n",
    "    \n",
    "    if eventId in eventIndex:# 若这个活动在训练或测试集中出现，python3 不再支持has_key方法，用in\n",
    "        i = eventIndex[eventId]\n",
    "        \n",
    "        #特征编码，这里只是简单处理，其实开始时间，地点等信息很重要\n",
    "        #将编码后的数据写入活动基本信息矩阵\n",
    "        eventPropMatrix[i, 0] = FE.getJoinedYearMonth(cols[2]) # start_time\n",
    "        eventPropMatrix[i, 1] = FE.getFeatureHash(cols[3].encode('utf-8')) # city\n",
    "        eventPropMatrix[i, 2] = FE.getFeatureHash(cols[4].encode('utf-8')) # state\n",
    "        eventPropMatrix[i, 3] = FE.getFeatureHash(cols[5].encode('utf-8')) # zip\n",
    "        eventPropMatrix[i, 4] = FE.getFeatureHash(cols[6].encode('utf-8')) # country\n",
    "        eventPropMatrix[i, 5] = FE.getFloatValue(cols[7]) # lat\n",
    "        eventPropMatrix[i, 6] = FE.getFloatValue(cols[8]) # lng\n",
    "        \n",
    "        #将编码后的数据写入词频特征矩阵\n",
    "        for j in range(9,110):\n",
    "            eventContMatrix[i, j-9] = cols[j]\n",
    "fin.close()\n",
    "\n",
    "# 对活动基本信息数据进行正则化（L2）\n",
    "eventPropMatrix = normalize(eventPropMatrix, norm = 'l2', axis = 0, copy = False )\n",
    "sio.mmwrite('EV_eventPropMatrix', eventPropMatrix) # 保存处理后的活动基本信息矩阵\n",
    "\n",
    "# 对活动词频数据进行正则化（L2），并考虑对这部分词频特征进行聚类\n",
    "eventContMatrix = normalize(eventContMatrix, norm = 'l2', axis = 0, copy = False )\n",
    "sio.mmwrite('EV_eventContMatrix', eventContMatrix) # 保存处理后的活动词频矩阵       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算活动之间的相似度\n",
    "# 基于活动基本信息矩阵、活动词频矩阵，计算活动对的相似性\n",
    "\n",
    "eventPropSim = ss.dok_matrix((n_events, n_events))\n",
    "eventContSim = ss.dok_matrix((n_events, n_events))\n",
    "\n",
    "# 读取在测试集和训练集中出现的活动对\n",
    "uniqueEventPairs = pickle.load(open('PE_uniqueEventPairs.pkl', 'rb'))\n",
    "\n",
    "for i, j in uniqueEventPairs:\n",
    "    # 对活动基本信息特征采用Person相关系数作为相似度\n",
    "    if (i, j) not in eventPropSim:\n",
    "        epsim = ssd.correlation(eventPropMatrix.getrow(i).todense(),eventPropMatrix.getrow(j).todense())\n",
    "        eventPropSim[i,j] = epsim\n",
    "        eventPropSim[j,i] = epsim\n",
    "            \n",
    "    # 对词频特征，采用余弦相似度\n",
    "    if (i, j) not in eventContSim:\n",
    "        ecsim = ssd.correlation(eventContMatrix.getrow(i).todense(),eventContMatrix.getrow(j).todense())\n",
    "        eventContSim[i, j] = ecsim\n",
    "        eventContSim[j, i] = ecsim\n",
    "\n",
    "sio.mmwrite('EV_eventPropSim', eventPropSim)\n",
    "sio.mmwrite('EV_eventContSim', eventContSim)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
